const fs = require("fs");
const path = require("node:path");
const schedule = require("node-schedule");
const moment = require("moment");
const colors = require('colors');

const EventEmitter = require("node:events");
const { startPlaywrightNew} = require("./use_crawlee_playwright");
const {transform} = require("./excel_processor");
let config = require("./config/scrapy.config.json");
const { TaskEndEvent } = require("./common");


const eventEmitter = new EventEmitter();

let defaultZipCode = "90001";

const start = async ()=>{

    let batchDir = await setupFolder();

    let querys = require("../input/query.json");

    let currentTaskCount = Math.max(1,config.concurrentTaskNumber);
    
    let taskGroups = [];
    
    let tasks = []
    let total = 0;

    for (let i=0; i<querys.items.length; i++){
        
        let query = querys.items[i];
        let zipcode = query["zipcode"]||defaultZipCode;
        let keywords = query['keywords']||[];
        /**@type Array */
        if(taskGroups.length > 0){
            let lastTasks = taskGroups[taskGroups.length - 1]
            if (lastTasks.length < currentTaskCount){
                tasks = lastTasks;
            }
        }
        for(let j=0; j<keywords.length; j++){
            total ++;
            let keyword = keywords[j];
            //build a task
            let item = {zipcode,keyword,taskNumber:total}

            tasks.push(item)
            
            if(tasks.length == currentTaskCount || (i==querys.items.length -1 && j== keywords.length -1)){
                taskGroups.push({
                    zipcode,
                    tasks:[...tasks]
                });
                tasks = []
            }
        }
    }    

    for (let idx = 0; idx< taskGroups.length; idx++){
        var group = taskGroups[idx];
        let zipcode = taskGroups[idx].zipcode;
        console.log(`start task group for batch ${idx+1}`);
        //make the directory for  zip code
        let zipDir = path.resolve(batchDir,zipcode);
        if(!fs.existsSync(zipDir)){
            fs.mkdirSync(zipDir);
        }

        var allTasks = group.tasks;

        await startPlaywrightNew(allTasks,zipDir,zipcode)
    }

    eventEmitter.on(TaskEndEvent,()=>{
        const outputFormat = ".csv";
        const csvfiles = []
        const readAll = (fp)=>{
            const filenames = fs.readdirSync(fp,{"recursive":true})
            filenames.forEach((f)=>{
                const p = path.join(fp,f);
                if(fs.statSync(p).isDirectory()){
                    readAll(p)
                }else{
                    if(p.endsWith(outputFormat) && !csvfiles.includes(p)){
                        csvfiles.push(p);
                    }
                }
            })    
        }
        readAll(batchDir);
        const outputs = csvfiles.filter((name)=> name.endsWith(outputFormat) && fs.statSync(path.resolve(batchDir,name)).isFile())
        console.log(`expect ${total}, actual ${outputs.length}`.yellow)
        if(outputs.length >= total){
            console.log(colors.green(`--------------------------------END-------------------------------------`));
        }else{
            console.log(colors.red(`-----------------------END But Missing some items -----------------------`));
            console.clear();
        }    
    })
}

const setupFolder = async()=>{
    try{
        await transform();
    }catch(e){
        return        
    }

    let day =  moment().format("yyyy_MM_DD");
    
    let outputDir = path.resolve(__dirname,'..',"output",day);

    if(!fs.existsSync(outputDir)){
        fs.mkdirSync(outputDir,{recursive:true})
    }
    const subdirs = fs.readdirSync(outputDir,{recursive:false});

    const dirnumbers = subdirs.map((num=>parseInt(num))).filter((num)=>!isNaN(num))

    let index = 1;

    if(subdirs.length > 0){
       index = Math.max(...dirnumbers,index)+1
    }

    var batchDir = path.resolve(outputDir,index.toString())

    if(!fs.existsSync(batchDir)){
        fs.mkdirSync(batchDir,{recursive:true});
    }

    return batchDir
}


start();
/* * * * * */
//second minute hour day month (day of week)

//轮询任务
// schedule.scheduleJob("scrapy task","0 20 14 * * *",()=>{
//     console.log("start scapy task.")
//     start({usePlaywright:true})
// })
